ANALYSIS OF CUSTOMER LOAN REPAYMENT HISTORY AND
PREDICTING CREDIT RISK SCORE & CUSTOMER SEGMENTATION
BY - SHAMBHURAJ BARGE¶
Agenda
In this hackhathon,I am going to build a machine learning model on the finantial dataset to predict CreditRiskScore of the home loan borrowers , which is used by many housing finance company to find the quantum of additional finance which can be allowed based on the mortgage. . The credit risk scores are used to evaluate the potential risk posed by lending money to consumers and in turn they can take decisions to mitigate losses due to bad debt..
we have two datasets provides information about demographics and finantial payment history borrowers .
Objectives
to do exploratory Data Analysis using visualizations
to build the analytical framework to predict the credit risk score of each customer using the payment history and other demographic features.
to segment the customers which will help to understand their sensitivity to the interest rate or product features to improve retention ie., to reduce the default or transfer of loan to competition and to understand how to structure offers for better revenue.
Importing The Necessary Libraries
In [390]:
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
get_ipython().magic('matplotlib inline')
import os
from imblearn.datasets import fetch_datasets
from kmeans_smote import KMeansSMOTE
from collections import Counter
from imblearn.datasets import fetch_datasets
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from imblearn.pipeline import make_pipeline as make_pipeline_imb
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import NearMiss
from imblearn.metrics import classification_report_imbalanced
from sklearn.metrics import precision_score, recall_score, f1_score, roc_auc_score, accuracy_score, classification_report
from sklearn.ensemble import RandomForestClassifier
import numpy as np
from imblearn.over_sampling import SMOTE
from math import *
import pandas as pd
import numpy as np
import tensorflow as tf
import warnings
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.preprocessing import Imputer, StandardScaler
import matplotlib.pyplot as plt
from matplotlib import gridspec
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization, Activation
from keras.optimizers import Adam
from keras import backend as K
from tensorflow.python.client import device_lib
import lightgbm as lgb
import catboost as cb
import xgboost as xgb
import seaborn as sns
sns.set_style("whitegrid")
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 200)
warnings.filterwarnings('ignore')
from pathlib import Path
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_samples, silhouette_score
from sklearn import preprocessing, model_selection, metrics, feature_selection
from sklearn.model_selection import GridSearchCV, learning_curve
from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix
from sklearn import neighbors, linear_model, svm, tree, ensemble
from sklearn.ensemble import AdaBoostClassifier
from sklearn.decomposition import PCA
from IPython.display import display, HTML
import plotly.graph_objs as go
from plotly.offline import init_notebook_mode,iplot
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
get_ipython().magic('matplotlib inline')
import os
from imblearn.datasets import fetch_datasets
from kmeans_smote import KMeansSMOTE
from collections import Counter
from imblearn.datasets import fetch_datasets
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from imblearn.pipeline import make_pipeline as make_pipeline_imb
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import NearMiss
from imblearn.metrics import classification_report_imbalanced
from sklearn.metrics import precision_score, recall_score, f1_score, roc_auc_score, accuracy_score, classification_report
from sklearn.ensemble import RandomForestClassifier
import numpy as np
from imblearn.over_sampling import SMOTE
from math import *
import pandas as pd
import numpy as np
import pickle
import tensorflow as tf
import warnings
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.preprocessing import Imputer, StandardScaler
import matplotlib.pyplot as plt
from matplotlib import gridspec
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization, Activation
from keras.optimizers import Adam
from keras import backend as K
from tensorflow.python.client import device_lib
import lightgbm as lgb
import catboost as cb
import xgboost as xgb
import seaborn as sns
sns.set_style("whitegrid")
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 200)
warnings.filterwarnings('ignore')
from pathlib import Path
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_samples, silhouette_score
from sklearn import preprocessing, model_selection, metrics, feature_selection
from sklearn.model_selection import GridSearchCV, learning_curve
from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix
from sklearn import neighbors, linear_model, svm, tree, ensemble
from sklearn.ensemble import AdaBoostClassifier
from sklearn.decomposition import PCA
from IPython.display import display, HTML
import plotly.graph_objs as go
from plotly.offline import init_notebook_mode,iplot
import warnings
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
import numpy as np
import pandas as pd
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
from scipy.stats import norm, skew
from sklearn import preprocessing
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.linear_model import ElasticNetCV, ElasticNet
from xgboost import XGBRegressor, plot_importance
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import StratifiedKFold
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
Set Path
In [391]:
os.chdir("F:\insofe\PHD\data\ml")
In [392]:
pwd()
Out[392]:
Creating a Pandas DataFrame from a CSV file
In [393]:
#raw train data 1
demographic_raw_dt=pd.read_excel("Train-1557019772293.xlsx")
#raw train data 2
payment_raw_dt=pd.read_excel("Train_PaymentHistory-1557019802161.xlsx")
# raw test demo data
test_demo_dt=pd.read_excel("Test-1557294637283.xlsx")
# raw test payment data
test_payment_dt=pd.read_excel("Test_PaymentHistory-1557294848030.xlsx")
# sample submission file
sample_file=pd.read_csv('samplesubmission-1557545918238.csv')
Data Exploration/Analysis¶
- dimension of data set
In [394]:
print("demographic dataset has {} samples with {} features each.".format(*demographic_raw_dt.shape))
print ("payment dataset has {} samples with {} features each.".format(*payment_raw_dt.shape))
- describing data
demographic data¶
In [395]:
demographic_raw_dt.head()
Out[395]:
In [396]:
demographic_raw_dt.columns
Out[396]:
In [397]:
demographic_raw_dt.dtypes
Out[397]:
payment data¶
In [398]:
payment_raw_dt.head(30)
Out[398]:
In [399]:
payment_raw_dt.dtypes
Out[399]:
Columns in the dataset
CustomerID - unique no
Current_Instalment_Sequence -
Starting_Instalment
Maturity_Period-he time between when the bond is issued and when it matures in months.
Current_Outstanding
Current_Loan_to_Appraisedvalu_Percent
CurrentInterestrate
RealEstate_Current_Inflation - property rate in percentage
GDP -is defined as the market value of the goods and services produced by a country.
UnemploymentRate
Asset_type
Urban_Development
Villa_House
Investment_SelfOccupied-'Self Occupancy', 'Investment'
Starting_outstanding - total debt
Starting_Loan_to_Appraisedvalu_Percent -LTV ratio is 75% or lower, you could get a lower rate, because the loan is seen as less risky to the lender. If the value of the home increases after you close on your home purchase, you may be able to refinance to a lower interest rate.
StartingInterestrate
RealEstate_Starting_Inflation
Payment_Status -'Non-Payoff/Non-Default', 'Default', 'Payoff'
Salary
ProfessionalLicensure
UtilitySpending
eCommerceAccount
SocialMediaAccount
DOB
NoOfProperties- total properties of borrowers.
CreditRiskScore -calculate the creditworthiness of borrowers. score them bet 300-900. need more than 650 to egligible for loan.
Convert DOB into Age
In [400]:
demographic_raw_dt['DOB']=demographic_raw_dt['DOB'].astype('datetime64')
demographic_raw_dt['age'] = (pd.to_datetime('now') - demographic_raw_dt['DOB']).astype('<m8[Y]')
demographic_raw_dt['age'] = demographic_raw_dt['age'].astype('int')
Remove DOB
In [401]:
demographic_raw_dt=demographic_raw_dt.drop(axis=1,columns='DOB')
combine two dataset on primary key that is CustomerID for doing further analysis¶
In [402]:
merge_dt=payment_raw_dt.merge(demographic_raw_dt, left_on='CustomerID', right_on='CustomerID')
In [403]:
merge_dt.columns
Out[403]:
In [404]:
merge_dt.shape
Out[404]:
In [405]:
merge_dt.head(5)
Out[405]:
In [406]:
merge_dt.info()
In [407]:
merge_dt.describe(include='all').T.sort_values("count")
Out[407]:
- Observation - ProfessionalLicensure,eCommerceAccount,NoOfProperties are categorical type so need to change its data types
- observation -CreditRiskScore is target columns and we need to remove before to build the clusters
In [408]:
print(merge_dt.ProfessionalLicensure.unique())
print(merge_dt.eCommerceAccount.unique())
print(merge_dt.NoOfProperties.unique())
In [409]:
merge_dt['ProfessionalLicensure']=merge_dt['ProfessionalLicensure'].astype('object')
merge_dt['eCommerceAccount']=merge_dt['eCommerceAccount'].astype('object')
merge_dt['NoOfProperties']=merge_dt['NoOfProperties'].astype('object')
In [410]:
merge_dt.Urban_Development.unique()
Out[410]:
In [411]:
merge_dt.describe(include=['O'])
Out[411]:
In [412]:
merge_dt.describe(include=['float32','float64','int64','int32'])
Out[412]:
- observation- all columns have correct data types
- observation - CreditRiskScore is max 830 means range is bet 300 to 900
Pre-Processing¶
Correlation Plot
In [413]:
import matplotlib.pyplot as plt
def data_corr(data):
correlation = data.corr()
fig, ax = plt.subplots(figsize=(15,15))
sns.heatmap(correlation, annot=True, cbar=True, cmap="RdYlGn")
num = list(merge_dt.select_dtypes(include=['float32','float64','int64','int32']).columns)
data_corr(merge_dt[num])
- observation -
- unemployement and real Estate current inflation shows high co-rrelation
- highest co-rrelation is 0.98
Check Missing Data
In [414]:
#function to find missing values
def miss_data(x):
total = x.isnull().sum().sort_values(ascending=False)
percent = (x.isnull().sum()/x.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data.index.name = 'column_names'
missing_data.reset_index(inplace=True)
sns.set(style="whitegrid")
sns.set(rc={'figure.figsize':(20,8.27)})
sns.barplot(missing_data.column_names,missing_data.Percent, alpha=0.9)
print(plt.title('missing data plot'))
print(plt.ylabel('percentage of missing data', fontsize=12))
print(plt.xlabel('column names', fontsize=12))
print(plt.show())
print(missing_data)
In [415]:
miss_data(x=merge_dt)
there is not any NA's present¶
Remove Duplicates
In [416]:
print('Entrées dupliquées: {}'.format(merge_dt.duplicated().sum()))
merge_dt.drop_duplicates(inplace = True)
Feature Engineering¶
Here we can generate three new features from outstanding columns¶
In [417]:
merge_dt['Appraisal_value']=merge_dt['Starting_outstanding']/(merge_dt['Starting_Loan_to_Appraisedvalu_Percent']/100)
In [418]:
merge_dt['current_Appraisal_value']=merge_dt['Current_Outstanding']/(merge_dt['Current_Loan_to_Appraisedvalu_Percent']/100)
In [419]:
merge_dt['remaining_outstanding']= merge_dt['Starting_outstanding'] - merge_dt['Current_Outstanding']
In [420]:
new_data=merge_dt[['CustomerID', 'Starting_Instalment','Maturity_Period','Asset_type',
'Urban_Development', 'Villa_House', 'Investment_SelfOccupied',
'Starting_outstanding', 'Starting_Loan_to_Appraisedvalu_Percent',
'StartingInterestrate', 'RealEstate_Starting_Inflation',
'age', 'Salary', 'ProfessionalLicensure','Payment_Status',
'UtilitySpending', 'eCommerceAccount', 'SocialMediaAccount','Appraisal_value',
'NoOfProperties', 'CreditRiskScore']]
In [421]:
new_data=new_data.drop_duplicates()
Create Credit Class
create credit class on credit score for visualization¶
In [422]:
credit_rate =[]
for i in range(len(new_data.CreditRiskScore)):
if new_data.CreditRiskScore.iloc[i] > 650:
credit_rate.append("high_score")
elif new_data.CreditRiskScore.iloc[i] < 650 and new_data.CreditRiskScore.iloc[i] > 450 :
credit_rate.append("Medium_score")
else:
credit_rate.append("low_score")
In [423]:
new_data['credit_class']=credit_rate
EDA¶
Univariate Analysis
Write function to plot histogram¶
In [424]:
def histplot(df):
cols = list(df.select_dtypes(include=['float32','float64','int64','int32']).columns)
for i in cols:
carrier_count= df[i].value_counts()
sns.set(style="darkgrid")
sns.distplot(new_data[i])
plt.title('Frequency Distribution of'+i)
plt.ylabel('Number of Occurrences', fontsize=12)
plt.xlabel(i, fontsize=12)
print(plt.show())
In [425]:
histplot(new_data)
In [426]:
sns.distplot(merge_dt['Current_Loan_to_Appraisedvalu_Percent'])
Out[426]:
In [427]:
sns.distplot(merge_dt['remaining_outstanding'])
Out[427]:
In [428]:
sns.distplot(merge_dt['UnemploymentRate'])
Out[428]:
In [429]:
sns.distplot(merge_dt['GDP'])
Out[429]:
In [430]:
sns.distplot(merge_dt['CurrentInterestrate'])
Out[430]:
In [431]:
sns.distplot(merge_dt['Current_Outstanding'])
Out[431]:
In [432]:
sns.distplot(merge_dt['Starting_Instalment'])
Out[432]:
In [433]:
sns.distplot(merge_dt['Starting_Instalment'])
Out[433]:
In [434]:
sns.distplot(merge_dt['Current_Instalment_Sequence'])
Out[434]:
observation- most of data is skewd need log or square root transformation before applying the model¶
Write function to plot categorical data¶
In [435]:
def distributionplot(df):
cols = list(df.select_dtypes(include=['object']).columns)
for i in cols:
carrier_count= df[i].value_counts()
sns.set(style="darkgrid")
sns.barplot(carrier_count.index, carrier_count.values, alpha=0.9)
plt.title('Frequency Distribution of'+i)
plt.ylabel('Number of Occurrences', fontsize=12)
plt.xlabel(i, fontsize=12)
print(plt.show())
In [436]:
merge_data=merge_dt
merge_data=merge_data.drop(axis=1,columns='CustomerID')
distributionplot(merge_data)
observation- many loan payments are late paid¶
Bivariate Analysis
In [437]:
# Standard plotly imports
import plotly.plotly as py
import plotly.graph_objs as go
from plotly.offline import iplot, init_notebook_mode
# Using plotly + cufflinks in offline mode
import cufflinks
cufflinks.go_offline(connected=True)
init_notebook_mode(connected=True)
In [438]:
demographic_raw_dt.pivot(columns='age', values='CreditRiskScore').iplot(
kind='box',xTitle='age',
yTitle='CreditRiskScore',
title='CreditRiskScore Distribution by age ')
- Observation- the most of people who get the loan having age bet 35 to 50
- range bet 36 - 40 age group has very high borrowers because of most of people are finantial strong bet this age group after 50 bank are less interested to give the loan. i think mostly guys are retired from the job."
- borrowers of age above 40 has very high credit score
In [439]:
color_seq = []
for i in new_data.SocialMediaAccount:
if i =='Yes':
color_seq.append('red')
else:
color_seq.append('green')
In [440]:
plt.figure(figsize = (10,7))
plt.scatter(x = new_data.Salary,y = new_data.Maturity_Period,color = color_seq)
plt.title('Maturity_Period vs Salary')
plt.xlabel('Salary')
plt.ylabel('Maturity_Period')
plt.show()
observation - we can here three dots having salary more than 200000 are the outliers¶
In [441]:
new_data.iplot(
x='Maturity_Period',
y='age',
# Specify the category
categories='credit_class',
xTitle='Maturity_Period',
yTitle='age',
title='Maturity_Period vs age by credit_class')
observation-¶
- 1) borrowers who are bet age group 35-40 have maturity period is 122-155 and all are having low credit score
- 2) age group more than 40 are belong to high score category
In [442]:
new_data.iplot(
x='Starting_Instalment',
y='StartingInterestrate',
# Specify the category
categories='credit_class',
xTitle='Starting_Instalment',
yTitle='StartingInterestrate',
title='StartingInterestrate vs Starting_Instalment by credit_class ')
observation-¶
- 1) medium score people have mostly starting instalment bet 15-35
Univariate Analysis[OUTLIERS]
define function to plot boxplot¶
In [443]:
def boxplot(df):
cols = list(df.select_dtypes(include=['float32','float64','int64','int32']).columns)
for i in cols:
carrier_count= df[i].value_counts()
sns.set(style="darkgrid")
sns.boxplot(df[i])
plt.title('Frequency Distribution of'+i)
plt.ylabel('Number of Occurrences', fontsize=12)
plt.xlabel((i), fontsize=12)
print(plt.show())
#'float32','float64','int64','int32'
In [444]:
boxplot(merge_dt)
#sns.boxplot(x=new_data['Starting_Instalment'])
Observation -¶
- 1) Current interest rate column has one outlier as rate more than 35
- 2) GDP column has one value greater than 4 and 3 are below 0
- 3) starting outstanding column has one value at 8000000
- 4) salary column has three outliers having salary more than 200000
- 5) one Appraisal value is more than 1.4 e7
Remove Outliers
Current_Outstanding¶
In [445]:
merge_dt.loc[(merge_dt["Current_Outstanding"] >7000000 )]
Out[445]:
In [446]:
merge_dt=merge_dt[merge_dt.CustomerID != 'C15226']
Salary¶
In [447]:
merge_dt.loc[(merge_dt["Salary"] > 200000 )]
Out[447]:
In [448]:
merge_dt=merge_dt[merge_dt.CustomerID !='C18942']
merge_dt=merge_dt[merge_dt.CustomerID !='C19970']
merge_dt=merge_dt[merge_dt.CustomerID !='C21710']
Appraisal_value¶
In [449]:
merge_dt.loc[(merge_dt["Appraisal_value"] > 8.097117e+06 )]## showing nothing means its already dropped
Out[449]:
define function to remove outliers in GDP column¶
In [450]:
def remove_outlier(df_in, col_name):
q1 = df_in[col_name].quantile(0.25)
q3 = df_in[col_name].quantile(0.75)
iqr = q3-q1 #Interquartile range
fence_low = q1-1.5*iqr
fence_high = q3+1.5*iqr
df_out = df_in.loc[(df_in[col_name] > fence_low) & (df_in[col_name] < fence_high)]
return df_out
In [451]:
merge_dt=remove_outlier(df_in=merge_dt,col_name='GDP')
In [452]:
merge_dt=remove_outlier(df_in=merge_dt,col_name='Current_Instalment_Sequence')
In [453]:
merge_dt.shape
Out[453]:
CLUSTER ANALYSIS
In [454]:
cluster_data=merge_dt
Drop CustomerID¶
In [455]:
cluster_data=cluster_data.drop(axis=1,columns="CustomerID")
In [456]:
#cluster_data =merge_dt.set_index("CustomerID") #assign customer id as index to dataset to visualize the clusters
In [457]:
cluster_data.head(2)
Out[457]:
In [458]:
cluster_data=cluster_data.drop(axis=1,columns='CreditRiskScore')
In [459]:
cluster_data=cluster_data.fillna(value=cluster_data.current_Appraisal_value.mean())
change datatypes¶
In [460]:
cluster_data['Appraisal_value']=cluster_data['Appraisal_value'].astype('int')
In [461]:
cluster_data['current_Appraisal_value']= cluster_data['current_Appraisal_value'].astype('int')
In [462]:
cluster_data['NoOfProperties']= cluster_data['NoOfProperties'].astype('object')
List out categorical and numerical data¶
In [463]:
cat_cols = list(cluster_data.select_dtypes(include=['object']).columns)
num_cols = list(cluster_data.select_dtypes(include=['float64','float32','int32','int64']).columns)
In [464]:
cat_cols
Out[464]:
Dummification
In [465]:
dummies_df=pd.get_dummies(cluster_data[cat_cols],drop_first=True)
In [466]:
numerical_data = cluster_data[num_cols]
In [467]:
numerical_data['index']=range(233335)
dummies_df['index']=range(233335)
In [468]:
final_data1=numerical_data.merge(dummies_df, left_on='index', right_on='index')
final_data1=final_data1.drop(axis=1,columns='index')
In [500]:
final_data1.to_csv('final_data.csv')
Find optimum no of K¶
In [469]:
train_matrix1 = final_data1.as_matrix()
In [470]:
Sum_of_squared_distances = []
K = range(1,10)
for k in K:
km = KMeans(n_clusters=k)
km = km.fit(train_matrix1)
Sum_of_squared_distances.append(km.inertia_)
In [471]:
plt.plot(K, Sum_of_squared_distances, 'bx-')
plt.xlabel('k')
plt.ylabel('Sum_of_squared_distances')
plt.title('Elbow Method For Optimal k')
plt.show()
observation- after K- 4 the SSD reamain constant¶
create K-means cluster with k - 4
In [ ]:
kmeans1 = KMeans(n_clusters = 4)
kmeans1.fit(train_matrix1)
In [473]:
clusters=kmeans1.labels_
In [474]:
cluster_class=clusters.tolist()
In [475]:
len(cluster_class)
Out[475]:
In [476]:
final_data1.shape
Out[476]:
In [477]:
cluster_dt=merge_dt
In [478]:
cluster_dt['Cluster_class']=cluster_class
Customer Segmentation¶
In [479]:
cluster_dt.Cluster_class.value_counts()
Out[479]:
0bservation- cluster 2 is small cluster than others
create credit class
In [47]:
credit_rate =[]
for i in range(len(cluster_dt.CreditRiskScore)):
if cluster_dt.CreditRiskScore.iloc[i] > 650:
credit_rate.append("high_score")
elif cluster_dt.CreditRiskScore.iloc[i] < 650 and cluster_dt.CreditRiskScore.iloc[i] > 450 :
credit_rate.append("Medium_score")
else:
credit_rate.append("low_score")
In [48]:
cluster_dt['credit_class']=credit_rate
create seperate dataframe for each cluster
In [49]:
cust_seg_1=cluster_dt[cluster_dt.Cluster_class == 0]
cust_seg_2=cluster_dt[cluster_dt.Cluster_class == 1]
cust_seg_3=cluster_dt[cluster_dt.Cluster_class == 2]
cust_seg_4=cluster_dt[cluster_dt.Cluster_class == 3]
In [60]:
################### 1 #########################################
cust_high_1=cust_seg_1[cust_seg_1.credit_class == 'high_score']
cust_medium_1=cust_seg_1[cust_seg_1.credit_class == 'Medium_score']
cust_low_1=cust_seg_1[cust_seg_1.credit_class == 'low_score']
################### 2 ###########################################
cust_high_2=cust_seg_2[cust_seg_2.credit_class == 'high_score']
cust_medium_2=cust_seg_2[cust_seg_2.credit_class == 'Medium_score']
cust_low_2=cust_seg_2[cust_seg_2.credit_class == 'low_score']
################### 3 ###########################################
cust_high_3=cust_seg_3[cust_seg_3.credit_class == 'high_score']
cust_medium_3=cust_seg_3[cust_seg_3.credit_class == 'Medium_score']
cust_low_3=cust_seg_3[cust_seg_3.credit_class == 'low_score']
################### 4 ###########################################
cust_high_4=cust_seg_4[cust_seg_4.credit_class == 'high_score']
cust_medium_4=cust_seg_4[cust_seg_4.credit_class == 'Medium_score']
cust_low_4=cust_seg_4[cust_seg_4.credit_class == 'low_score']
In [52]:
cluster_dt.credit_class.value_counts()
Out[52]:
customer seg -2¶
In [63]:
score_high = cust_high_1['CurrentInterestrate']
score_medium = cust_medium_1['CurrentInterestrate']
score_low = cust_low_1['CurrentInterestrate']
legend = ['High', 'Medium','Low']
plt.hist([score_high, score_medium ,score_low], color=['orange', 'green','blue'])
plt.xlabel("CurrentInterestrate")
plt.ylabel("Frequency")
plt.legend(legend)
#plt.xticks(range(0, 7))
#plt.yticks(range(1, 20))
plt.title('Histogram of CurrentInterestrate for clust 1')
plt.show()
customer seg -3¶
In [62]:
score_high = cust_high_2['CurrentInterestrate']
score_medium = cust_medium_2['CurrentInterestrate']
score_low = cust_low_2['CurrentInterestrate']
legend = ['High', 'Medium','Low']
plt.hist([score_high, score_medium ,score_low], color=['orange', 'green','blue'])
plt.xlabel("CurrentInterestrate")
plt.ylabel("Frequency")
plt.legend(legend)
#plt.xticks(range(0, 7))
#plt.yticks(range(1, 20))
plt.title('Histogram of CurrentInterestrate for clust 2')
plt.show()
customer seg -4¶
In [64]:
score_high = cust_high_3['CurrentInterestrate']
score_medium = cust_medium_3['CurrentInterestrate']
score_low = cust_low_3['CurrentInterestrate']
legend = ['High', 'Medium','Low']
plt.hist([score_high, score_medium ,score_low], color=['orange', 'green','blue'])
plt.xlabel("CurrentInterestrate")
plt.ylabel("Frequency")
plt.legend(legend)
#plt.xticks(range(0, 7))
#plt.yticks(range(1, 20))
plt.title('Histogram of CurrentInterestrate for clust 3')
plt.show()
customer seg -1¶
In [65]:
score_high = cust_high_4['CurrentInterestrate']
score_medium = cust_medium_4['CurrentInterestrate']
score_low = cust_low_4['CurrentInterestrate']
legend = ['High', 'Medium','Low']
plt.hist([score_high, score_medium ,score_low], color=['orange', 'green','blue'])
plt.xlabel("CurrentInterestrate")
plt.ylabel("Frequency")
plt.legend(legend)
#plt.xticks(range(0, 7))
#plt.yticks(range(1, 20))
plt.title('Histogram of CurrentInterestrate for clust 3')
plt.show()
In [41]:
cust_seg_1[['CurrentInterestrate','CreditRiskScore']].describe()
Out[41]:
In [43]:
cust_seg_2[['CurrentInterestrate','CreditRiskScore']].describe()
Out[43]:
In [44]:
cust_seg_3[['CurrentInterestrate','CreditRiskScore']].describe()
Out[44]:
In [46]:
cust_seg_4[['CurrentInterestrate','CreditRiskScore']].describe()
Out[46]:
K-means clustering on PCA Data
In [239]:
pca = PCA()
pca.fit(train_matrix2)
pca_samples = pca.transform(scaled_matrix)
In [243]:
fig, ax = plt.subplots(figsize=(14, 5))
sns.set(font_scale=1)
plt.step(range(matrix.shape[1]), pca.explained_variance_ratio_.cumsum(), where='mid',
label='cumulative explained variance')
sns.barplot(np.arange(1,matrix.shape[1]+1), pca.explained_variance_ratio_, alpha=0.5, color = 'g',
label='individual explained variance')
plt.xlim(0, 31)
ax.set_xticklabels([s if int(s.get_text())%2 == 0 else '' for s in ax.get_xticklabels()])
plt.ylabel('Explained variance', fontsize = 14)
plt.xlabel('Principal components', fontsize = 14)
plt.legend(loc='best', fontsize = 13);
In [ ]:
X_normalized = StandardScaler().fit(X_cancer).transform(X_cancer)
pca = PCA(n_components = 2).fit(X_normalized)
pca_df = pca.transform(X_normalized)
In [97]:
Sum_of_squared_distances = []
K = range(1,10)
for k in K:
km = KMeans(n_clusters=k,init='k-means++',n_init=100)
km = km.fit(pca_df)
Sum_of_squared_distances.append(km.inertia_)
In [98]:
plt.plot(K, Sum_of_squared_distances, 'bx-')
plt.xlabel('k')
plt.ylabel('Sum_of_squared_distances')
plt.title('Elbow Method For Optimal k')
plt.show()
In [107]:
%matplotlib notebook
import numpy as np
import pandas as pd
import seaborn as sn
import matplotlib.pyplot as plt
from adspy_shared_utilities import plot_labelled_scatter
kmeans2 = KMeans(n_clusters = 3, random_state = 0)
kmeans2.fit(pca_df)
plot_labelled_scatter(pca_df, kmeans2.labels_,
['Cluster 1', 'Cluster 2', 'Cluster 3'])
BUILDING ML MODEL
Implemented Models ¶
7) Stacking-(Decision-Tree,SVM,Xg boost,RF,Light GBM,extra trees)
8) AdaBoost-AdaBoost)
11) XGBoost-XGBoost on PCA)
Also Used¶
In [65]:
merge_dt.columns
Out[65]:
Data Needs Grouping
In [66]:
merge_dt_1=merge_dt[['CustomerID','Current_Instalment_Sequence', 'Current_Outstanding','Current_Loan_to_Appraisedvalu_Percent','CurrentInterestrate',
'RealEstate_Current_Inflation','GDP', 'UnemploymentRate','current_Appraisal_value','remaining_outstanding']]
In [67]:
merge_dt_1.shape
Out[67]:
Data Don't Need Grouping
In [68]:
merge_dt_2=merge_dt[['CustomerID', 'Starting_Instalment','Maturity_Period','Asset_type',
'Urban_Development', 'Villa_House', 'Investment_SelfOccupied',
'Starting_outstanding', 'Starting_Loan_to_Appraisedvalu_Percent',
'StartingInterestrate', 'RealEstate_Starting_Inflation',
'age', 'Salary', 'ProfessionalLicensure',
'UtilitySpending', 'eCommerceAccount', 'SocialMediaAccount','Appraisal_value',
'NoOfProperties', 'CreditRiskScore']]
In [69]:
merge_dt_2=merge_dt_2.drop_duplicates()# drop duplicates from dataset
In [70]:
merge_dt_2.shape
Out[70]:
Function To Group The Data
First Approach
Build the model on whole dataset with out grouping and find the AVg of prediction at last for each group¶
Second Approach
Descriptive statistics involves summarizing and organizing the data so they can be easily understood. Descriptive statistics, unlike inferential statistics, seeks to describe the data¶
In [71]:
def feat_eng(data):
df = pd.DataFrame()
for col in data.columns:
if col in ['CustomerID']:
continue
df[col + '_mean'] = data.groupby(['CustomerID'])[col].mean()
df[col + '_median'] = data.groupby(['CustomerID'])[col].median()
df[col + '_max'] = data.groupby(['CustomerID'])[col].max()
df[col + '_min'] = data.groupby(['CustomerID'])[col].min()
df[col + '_std'] = data.groupby(['CustomerID'])[col].std()
#df[col + '_kurtosis'] = data.groupby(['CustomerID'])[col].kurtosis()
df[col + '_skew'] = data.groupby(['CustomerID'])[col].skew()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_kurtosis'] = data.groupby(['CustomerID'])[col].apply(lambda x: x.kurtosis())
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
#df[col + '_coeffvar'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.std(x) / np.mean(x))
#in statistics, the median absolute deviation (MAD) is a robust measure of the variablility of a univariate sample of quantitative data.
df[col + '_meanAD'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_mad'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.median(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.max(np.abs(x)))
df[col + '_abs_min'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.min(np.abs(x)))
df[col + '_abs_avg'] = (df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df
In [72]:
final_dt_1=feat_eng(data=merge_dt_1)
In [73]:
final_dt_1.head(10)
Out[73]:
In [74]:
final_dt_1['CustomerID']=final_dt_1.index
In [75]:
final_dt_1.head(10)
Out[75]:
In [76]:
final_dt_1.shape
Out[76]:
In [77]:
final_dt_1.describe(include='all')
Out[77]:
Join Two Column Merge dt 2 And final dt 1
In [78]:
train_data_1=final_dt_1.merge(merge_dt_2, left_on="CustomerID", right_on='CustomerID')
In [79]:
train_data_1.fillna(train_data_1['remaining_outstanding_maxtoMin'].median(),inplace=True)
In [80]:
train_data_1.isna().sum()
Out[80]:
In [81]:
train_data_1[['eCommerceAccount','ProfessionalLicensure']]=train_data_1[['eCommerceAccount','ProfessionalLicensure']].astype('object')
In [82]:
#num1_cols = list(train_data_1.select_dtypes(include=['float64','float32','int32','int64']).columns)
#cat1_cols = list(train_data_1.select_dtypes(include=['object']).columns)
categorical_features =train_data_1.select_dtypes(include=['object']).columns
numerical_features =train_data_1.select_dtypes(include=['float64','float32','int32','int64']).columns
Feature Engg On Payment Status Column
In [83]:
new=merge_dt[['CustomerID','Current_Instalment_Sequence', 'Current_Outstanding','Current_Loan_to_Appraisedvalu_Percent','CurrentInterestrate',
'RealEstate_Current_Inflation','GDP', 'UnemploymentRate','current_Appraisal_value','remaining_outstanding','Payment_Status']]
In [84]:
new.Payment_Status.unique()
Out[84]:
In [85]:
payment={'Non-Payoff/Non-Default': 2,'Payoff':4,'Default':0}
new['Payment_Status']=new['Payment_Status'].map(payment)
create payment_total_score¶
In [86]:
new['payment_total_score'] =[4]*new.shape[0]
create payment dataframe¶
In [87]:
payment_data=pd.DataFrame({'CustomerID':new.CustomerID,'Payment_Status':new.Payment_Status,'payment_total_score':new.payment_total_score})
define function to find z score of payment status¶
In [88]:
def payment_eng1(data):
df = pd.DataFrame()
for col in data.columns:
if col in ['CustomerID']:
continue
df[col + '_count'] = data.groupby(['CustomerID'])[col].count()
df[col + '_sum'] = data.groupby(['CustomerID'])[col].sum()
return df
In [89]:
payment_data=payment_eng1(data=payment_data)
In [90]:
payment_data.head(5)
Out[90]:
In [91]:
from scipy.stats import zscore
payment_percentile= payment_data.Payment_Status_sum/payment_data.payment_total_score_sum
payment_data['payment_z_score']=zscore(payment_percentile)
payment_data['payment_lenght']=payment_data.Payment_Status_count
In [92]:
print('size of train data',train_data_1.shape)
print('size of payment col',len(payment_percentile))
In [94]:
payment_data2=payment_data[['payment_z_score','payment_lenght']]
In [95]:
train_data_1=train_data_1.merge(payment_data2, left_on="CustomerID", right_on='CustomerID')
In [96]:
train_data_1.head(5)
Out[96]:
CreditRiskScore
In [97]:
sns.distplot(train_data_1['CreditRiskScore'] , fit=norm);
# Get the fitted parameters used by the function
(mu, sigma) = norm.fit(train_data_1['CreditRiskScore'])
print( '\n mu = {:.2f} and sigma = {:.2f}\n'.format(mu, sigma))
#Now plot the distribution
plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)],
loc='best')
plt.ylabel('Frequency')
plt.title('credit score distribution')
#Get also the QQ-plot
fig = plt.figure()
res = stats.probplot(train_data_1['CreditRiskScore'], plot=plt)
plt.show();
In [98]:
sns.distplot(np.log1p(train_data_1['CreditRiskScore']) , fit=norm);
# Get the fitted parameters used by the function
(mu, sigma) = norm.fit(np.log1p(train_data_1['CreditRiskScore']))
print( '\n mu = {:.2f} and sigma = {:.2f}\n'.format(mu, sigma))
#Now plot the distribution
plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)],
loc='best')
plt.ylabel('Frequency')
plt.title('log(credit score+1) distribution')
#Get also the QQ-plot
fig = plt.figure()
res = stats.probplot(np.log1p(train_data_1['CreditRiskScore']), plot=plt)
plt.show();
Correlation Plot
In [99]:
a=train_data_1[numerical_features]
corr_with_credit_score = a.corr()['CreditRiskScore'].sort_values(ascending=False)
plt.figure(figsize=(25,10))
corr_with_credit_score.drop('CreditRiskScore').plot.bar()
plt.show();
In [100]:
train_data_1['CreditRiskScore'] = np.log1p(train_data_1['CreditRiskScore'])
#log transform skewed numeric features:
numeric_feats = train_data_1.dtypes[train_data_1.dtypes != "object"].index
skewed_feats = train_data_1[numeric_feats].apply(lambda x: skew(x.dropna())) #compute skewness
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
train_data_1[skewed_feats] = np.log1p(train_data_1[skewed_feats])
In [101]:
train_data_1[numeric_feats] = train_data_1[numeric_feats].apply(lambda x:pd.to_numeric(x)) #
In [102]:
train_data_1.drop(axis=1,columns='CustomerID',inplace=True)
In [103]:
df=train_data_1
In [104]:
df.head(5)
Out[104]:
In [105]:
df = pd.get_dummies(df,drop_first=True)
df = df.fillna(df.mean())
In [106]:
df.isna().sum()
Out[106]:
Remove na From Dataset¶
In [107]:
import pandas as pd
def clean_dataset(df):
assert isinstance(df, pd.DataFrame), "df needs to be a pd.DataFrame"
df.dropna(inplace=True)
indices_to_keep = ~df.isin([np.nan, np.inf, -np.inf]).any(1)
return df[indices_to_keep].astype(np.float64)
In [108]:
df1=clean_dataset(df)
In [109]:
df1.shape
Out[109]:
Split Data
In [110]:
#X, y = df.drop(['CreditRiskScore'], axis = 1), df['CreditRiskScore']
#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
y= df1['CreditRiskScore']
X=df1.drop(['CreditRiskScore'], axis = 1)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)
BaseLine Model
1 MLR¶
In [112]:
import statsmodels.api as sm
import statsmodels.formula.api as smf
y=y_train
X=X_train
model = smf.OLS(y,X).fit()
predictions = model.predict(X)
print(model.summary())
Observation-R-squared and Adj. R-squared shows very high¶
In [113]:
test_prediction = model.predict(X_test)
In [115]:
print('Train r2 score: ', r2_score(predictions, y_train))
print('Test r2 score: ', r2_score(y_test, test_prediction))
train_mse1 = mean_squared_error(predictions, y_train)
test_mse1 = mean_squared_error(y_test, test_prediction)
train_rmse1 = np.sqrt(train_mse1)
test_rmse1 = np.sqrt(test_mse1)
print('Train RMSE: %.4f' % train_rmse1)
print('Test RMSE: %.4f' % test_rmse1)
observation- model build is so simple as RMSE are same¶
2 Elastic Net¶
In [116]:
from sklearn.linear_model import ElasticNet
from sklearn.model_selection import cross_val_score
from itertools import product
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring="mean_squared_error", cv = 5))
return(rmse)
In [117]:
alphas = [0.0005, 0.001, 0.01, 0.03, 0.05, 0.1]
l1_ratios = [1.5, 1.1, 1, 0.9, 0.8, 0.7, 0.5]
In [118]:
cv_elastic = [rmse_cv(ElasticNet(alpha = alpha, l1_ratio=l1_ratio)).mean()
for (alpha, l1_ratio) in product(alphas, l1_ratios)]
In [121]:
elastic = ElasticNet(alpha=0.0005, l1_ratio=0.9)
elastic.fit(X_train, y_train)
Out[121]:
In [122]:
#let's look at the residuals as well:
matplotlib.rcParams['figure.figsize'] = (6.0, 6.0)
preds = pd.DataFrame({"preds":elastic.predict(X_train), "true":y_train})
preds["residuals"] = preds["true"] - preds["preds"]
preds.plot(x = "preds", y = "residuals",kind = "scatter")
Out[122]:
In [123]:
rmse = np.sqrt(np.mean((preds['true']-preds['preds'])**2))
print ('RMSE: {0:.4f}'.format(rmse))
In [124]:
from sklearn.metrics import r2_score
print('R^2 train: %.3f' % r2_score(preds['true'], preds['preds']))
In [125]:
coef = pd.Series(elastic.coef_, index = X_train.columns)
In [126]:
imp_coef = pd.concat([coef.sort_values().head(50),
coef.sort_values().tail(10)])
In [127]:
feature_importance = pd.Series(index = X_train.columns, data = np.abs(elastic.coef_))
n_selected_features = (feature_importance>0).sum()
print('{0:d} features, reduction of {1:2.2f}%'.format(
n_selected_features,(1-n_selected_features/len(feature_importance))*100))
feature_importance.sort_values().tail(30).plot(kind = 'bar', figsize = (18,6))
Out[127]:
As we want least RMSE lets move on Boosting models
3 XG boost¶
In [130]:
xgb_model1 = XGBRegressor()
xgb_model1.fit(X_train, y_train, verbose=False)
Out[130]:
In [131]:
y_train_pred1 = xgb_model1.predict(X_train)
y_pred1 = xgb_model1.predict(X_test)
print('Train r2 score: ', r2_score(y_train_pred1, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred1))
train_mse1 = mean_squared_error(y_train_pred1, y_train)
test_mse1 = mean_squared_error(y_pred1, y_test)
train_rmse1 = np.sqrt(train_mse1)
test_rmse1 = np.sqrt(test_mse1)
print('Train RMSE: %.4f' % train_rmse1)
print('Test RMSE: %.4f' % test_rmse1)
In [132]:
xgb_model2 = XGBRegressor(n_estimators=1000)
xgb_model2.fit(X_train, y_train, early_stopping_rounds=5,
eval_set=[(X_test, y_test)], verbose=False)
y_train_pred2 = xgb_model2.predict(X_train)
y_pred2 = xgb_model2.predict(X_test)
print('Train r2 score: ', r2_score(y_train_pred2, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred2))
train_mse2 = mean_squared_error(y_train_pred2, y_train)
test_mse2 = mean_squared_error(y_pred2, y_test)
train_rmse2 = np.sqrt(train_mse2)
test_rmse2 = np.sqrt(test_mse2)
print('Train RMSE: %.4f' % train_rmse2)
print('Test RMSE: %.4f' % test_rmse2)
In [133]:
xgb_model3 = XGBRegressor(n_estimators=1000, learning_rate=0.05)
xgb_model3.fit(X_train, y_train, early_stopping_rounds=5,
eval_set=[(X_test, y_test)], verbose=False)
y_train_pred3 = xgb_model3.predict(X_train)
y_pred3 = xgb_model3.predict(X_test)
print('Train r2 score: ', r2_score(y_train_pred3, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred3))
train_mse3 = mean_squared_error(y_train_pred3, y_train)
test_mse3 = mean_squared_error(y_pred3, y_test)
train_rmse3 = np.sqrt(train_mse3)
test_rmse3 = np.sqrt(test_mse3)
print('Train RMSE: %.4f' % train_rmse3)
print('Test RMSE: %.4f' % test_rmse3)
In [134]:
xgb_model4 = XGBRegressor(n_estimators=100, learning_rate=0.08, gamma=0, subsample=0.75,
colsample_bytree=1, max_depth=7, n_jobs=-1)
xgb_model4.fit(X_train,y_train)
y_train_pred4 = xgb_model4.predict(X_train)
y_pred4 = xgb_model4.predict(X_test)
print('Train r2 score: ', r2_score(y_train_pred4, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred4))
train_mse4 = mean_squared_error(y_train_pred4, y_train)
test_mse4 = mean_squared_error(y_pred4, y_test)
train_rmse4 = np.sqrt(train_mse4)
test_rmse4 = np.sqrt(test_mse4)
print('Train RMSE: %.4f' % train_rmse4)
print('Test RMSE: %.4f' % test_rmse4)
ON LEADERBOARD SCORE- 117.35
In [148]:
feature_important = xgb_model4.feature_importances_
keys = list( X_train.columns)
values = list(feature_important)
data = pd.DataFrame(data=values, index=keys, columns=["score"]).sort_values(by = "score", ascending=True)
data.plot(kind='barh',figsize=(30,55),fontsize=25)
Out[148]:
I concluded that xgb_model4 is the best model.¶
In [158]:
from collections import OrderedDict
OrderedDict(sorted(xgb_model4.get_booster().get_fscore().items(), key=lambda t: t[1], reverse=True))
Out[158]:
In [164]:
most_relevant_features=list(dict((k, v) for k, v in xgb_model4.get_booster().get_fscore().items() if v >= 4).keys())
Build MOdel On Top Imp features¶
In [165]:
train_x=df[most_relevant_features]
train_y=df['CreditRiskScore']
In [166]:
X_train, X_test, y_train, y_test = train_test_split(train_x, train_y, test_size = 0.2, random_state = 106)
xgb_model5 = XGBRegressor(n_estimators=100, learning_rate=0.08, gamma=0, subsample=0.75,
colsample_bytree=1, max_depth=7, n_jobs=-1)
xgb_model5.fit(X_train, y_train, early_stopping_rounds=5,
eval_set=[(X_test, y_test)], verbose=False)
y_train_pred5 = xgb_model5.predict(X_train)
y_pred5 = xgb_model5.predict(X_test)
print('Train r2 score: ', r2_score(y_train_pred5, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred5))
train_mse5 = mean_squared_error(y_train_pred5, y_train)
test_mse5 = mean_squared_error(y_pred5, y_test)
train_rmse5 = np.sqrt(train_mse5)
test_rmse5 = np.sqrt(test_mse5)
print('Train RMSE: %.4f' % train_rmse5)
print('Test RMSE: %.4f' % test_rmse5)
Hyperparameter Tuning¶
In [167]:
xgb1 = XGBRegressor()
parameters = {'nthread':[4], #when use hyperthread, xgboost may become slower
'objective':['reg:linear'],
'learning_rate': [0.03, 0.05, 0.08,0.1,0.3], #so called `eta` value
'max_depth': [5, 6, 7],
'min_child_weight': [4,6],
'silent': [1],
'subsample': [0.3,0.5,0.7,0.9],
'colsample_bytree': [0.5,0.7,1],
'n_estimators': [500]}
xgb_grid = RandomizedSearchCV(xgb1,
parameters,
cv = 5,
n_jobs = 5,
verbose=True)
xgb_grid.fit(X_train, y_train,early_stopping_rounds=5,
eval_set=[(X_test, y_test)])
print(xgb_grid.best_score_)
print(xgb_grid.best_params_)
In [169]:
train_x=df[most_relevant_features]
train_y=df['CreditRiskScore']
X_train, X_test, y_train, y_test = train_test_split(train_x, train_y, test_size = 0.2, random_state = 0)
Build model on best parameters¶
In [174]:
xgb_model6 = XGBRegressor(subsample= 0.9, silent= 1,
nthread= 4, n_estimators= 500, min_child_weight= 4,
max_depth= 6, learning_rate= 0.05, colsample_bytree= 1)
xgb_model6.fit(X_train, y_train, early_stopping_rounds=5,
eval_set=[(X_test, y_test)], verbose=False)
y_train_pred6 = xgb_model6.predict(X_train)
y_pred5 = xgb_model6.predict(X_test)
print('Train r2 score: ', r2_score(y_train_pred5, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred5))
train_mse5 = mean_squared_error(y_train_pred5, y_train)
test_mse5 = mean_squared_error(y_pred5, y_test)
train_rmse5 = np.sqrt(train_mse5)
test_rmse5 = np.sqrt(test_mse5)
print('Train RMSE: %.4f' % train_rmse5)
print('Test RMSE: %.4f' % test_rmse5)
ON LEADERBOARD SCORE- 113.55
4 KNN¶
In [176]:
#X, y = df.drop(['CreditRiskScore'], axis = 1), df['CreditRiskScore']
#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
y= df1['CreditRiskScore']
X=df1.drop(['CreditRiskScore'], axis = 1)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)
In [177]:
from sklearn.preprocessing import MinMaxScaler
sc = MinMaxScaler(feature_range=(0, 1))
#sc = StandardScaler()
x_train = sc.fit_transform(X_train)
x_test = sc.transform(X_test)
In [178]:
#import required packages
from sklearn import neighbors
from sklearn.metrics import mean_squared_error
from math import sqrt
import matplotlib.pyplot as plt
%matplotlib inline
rmse_val = [] #to store rmse values for different k
for K in range(20):
K = K+1
model = neighbors.KNeighborsRegressor(n_neighbors = K)
model.fit(x_train, y_train) #fit the model
pred=model.predict(x_test) #make prediction on test set
error = sqrt(mean_squared_error(y_test,pred)) #calculate rmse
rmse_val.append(error) #store rmse values
print('RMSE value for k= ' , K , 'is:', error)
#plotting the rmse values against k values
curve = pd.DataFrame(rmse_val) #elbow curve
curve.plot()
Out[178]:
In [179]:
model = neighbors.KNeighborsRegressor(n_neighbors = 12)# best k value
model.fit(x_train, y_train) #fit the model
pred=model.predict(x_test)
knn_train=model.predict(x_train)
knn_test=model.predict(x_test)
print('Train r2 score: ', r2_score(knn_train, y_train))
print('Test r2 score: ', r2_score(y_test, knn_test))
train_mse5 = mean_squared_error(knn_train, y_train)
test_mse5 = mean_squared_error(knn_test, y_test)
train_rmse5 = np.sqrt(train_mse5)
test_rmse5 = np.sqrt(test_mse5)
print('Train RMSE: %.4f' % train_rmse5)
print('Test RMSE: %.4f' % test_rmse5)
ON LEADERBOARD SCORE- 119.63
5 CAT BOOST¶
In [181]:
#Creating a training set for modeling and validation set to check model performance
train_cat_dt=train_data_1
X = train_cat_dt.drop(['CreditRiskScore'], axis=1)
y = train_cat_dt.CreditRiskScore
from sklearn.model_selection import train_test_split
X_train, X_validation, y_train, y_validation = train_test_split(X, y, train_size=0.7, random_state=1234)
In [182]:
categorical_features_indices = np.where(X_train.dtypes == np.object)[0]
In [183]:
#importing library and building model
from catboost import CatBoostRegressor
model=CatBoostRegressor(iterations=1000, depth=3, learning_rate=0.1, loss_function='RMSE',random_seed=42)
model.fit(X_train, y_train,cat_features=categorical_features_indices,eval_set=(X_validation, y_validation),plot=True)
Out[183]:
6 Stacking¶
In [190]:
#X, y = df.drop(['CreditRiskScore'], axis = 1), df['CreditRiskScore']
#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
y= df1['CreditRiskScore']
X=df1.drop(['CreditRiskScore'], axis = 1)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)
In [191]:
from sklearn.preprocessing import MinMaxScaler
sc = MinMaxScaler(feature_range=(0, 1))
#sc = StandardScaler()
x_train = sc.fit_transform(X_train)
x_test = sc.transform(X_test)
In [196]:
import numpy as np
import pandas as pd
# data precession
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold
# model
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, AdaBoostRegressor
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
class Ensemble(object):
def __init__(self, n_splits, stacker, base_models):
self.n_splits = n_splits
self.stacker = stacker
self.base_models = base_models
def fit_predict(self, X, y, T):
X = np.array(X)
y = np.array(y)
T = np.array(T)
folds = list(KFold(n_splits=self.n_splits, shuffle=True, random_state=2016).split(X, y))
S_train = np.zeros((X.shape[0], len(self.base_models)))
S_test = np.zeros((T.shape[0], len(self.base_models)))
for i, clf in enumerate(self.base_models):
S_test_i = np.zeros((T.shape[0], self.n_splits))
for j, (train_idx, test_idx) in enumerate(folds):
X_train = X[train_idx]
y_train = y[train_idx]
X_holdout = X[test_idx]
y_holdout = y[test_idx]
print ("Fit Model %d fold %d" % (i, j))
clf.fit(X_train, y_train)
y_pred = clf.predict(X_holdout)[:]
S_train[test_idx, i] = y_pred
S_test_i[:, j] = clf.predict(T)[:]
S_test[:, i] = S_test_i.mean(axis=1)
# results = cross_val_score(self.stacker, S_train, y, cv=5, scoring='r2')
# print("Stacker score: %.4f (%.4f)" % (results.mean(), results.std()))
# exit()
self.stacker.fit(S_train, y)
res = self.stacker.predict(S_test)[:]
return res
# rf params
rf_params = {}
rf_params['n_estimators'] = 1000
rf_params['max_depth'] = 8
rf_params['min_samples_split'] = 100
rf_params['min_samples_leaf'] = 30
# xgb params
xgb_params = {}
xgb_params['n_estimators'] = 500
xgb_params['min_child_weight'] = 12
xgb_params['learning_rate'] = 0.12
xgb_params['max_depth'] = 6
xgb_params['subsample'] = 0.77
xgb_params['reg_lambda'] = 0.8
xgb_params['reg_alpha'] = 0.4
xgb_params['base_score'] = 0
#xgb_params['seed'] = 400
xgb_params['silent'] = 1
# lgb params
lgb_params = {}
lgb_params['n_estimators'] = 450
lgb_params['max_bin'] = 8
lgb_params['learning_rate'] = 0.037 # shrinkage_rate
lgb_params['metric'] = 'l1' # or 'mae'
lgb_params['sub_feature'] = 0.35
lgb_params['bagging_fraction'] = 0.85 # sub_row
lgb_params['bagging_freq'] = 40
lgb_params['num_leaves'] = 512 # num_leaf
lgb_params['min_data'] = 500 # min_data_in_leaf
lgb_params['min_hessian'] = 0.05 # min_sum_hessian_in_leaf
lgb_params['verbose'] = 0
lgb_params['feature_fraction_seed'] = 2
lgb_params['bagging_seed'] = 3
# XGB model
xgb_model = XGBRegressor(**xgb_params)
# lgb model
lgb_model = LGBMRegressor(**lgb_params)
# RF model
rf_model = RandomForestRegressor(**rf_params)
# ET model
et_model = ExtraTreesRegressor()
# SVR model
#SVM is too slow in more then 10000 set
#svr_model = SVR(kernel='rbf', C=1.0, epsilon=0.05)
# DecsionTree model
dt_model = DecisionTreeRegressor()
# AdaBoost model
ada_model = AdaBoostRegressor(learning_rate=0.1, loss='square', n_estimators=1000)
stack = Ensemble(n_splits=5,
stacker=LinearRegression(),
base_models=(rf_model,lgb_model, ada_model,et_model,dt_model,xgb_model))
In [197]:
y_pred_stack = stack.fit_predict(x_train, y_train, x_test)
In [201]:
#print('Train r2 score: ', r2_score(xgb_pred_train, y_train))
print('Test r2 score: ', r2_score(y_pred_stack, y_test))
#train_mse5 = mean_squared_error(xgb_pred_train, y_train)
test_mse5 = mean_squared_error(y_pred_stack, y_test)
#train_rmse5 = np.sqrt(train_mse5)
test_rmse5 = np.sqrt(test_mse5)
#print('Train RMSE: %.4f' % train_rmse5)
print('Test RMSE: %.4f' % test_rmse5)
ON LEADERBOARD SCORE- 116.6
we got least RMSE¶
7 AdaBoost¶
In [234]:
from sklearn.ensemble import AdaBoostRegressor
from sklearn.datasets import make_regression
X_train, y_train = make_regression(n_features=4, n_informative=2,
random_state=0, shuffle=False)
regr = AdaBoostRegressor(random_state=0, n_estimators=100,learning_rate=0.001)
regr.fit(X_train, y_train)
Out[234]:
In [235]:
from sklearn.ensemble import AdaBoostRegressor
ada2=AdaBoostRegressor(n_estimators=500,learning_rate=0.001,random_state=1)
score=np.mean(cross_val_score(ada2,X_train,y_train,scoring='neg_mean_squared_error',cv=5,n_jobs=1))
score
Out[235]:
Dimensionality Reduction¶
PCA¶
In [206]:
from sklearn.decomposition import PCA
#scaled_matrix=X_train.as_matrix
matrix = X_train.as_matrix()
scaler = StandardScaler()
scaler.fit(matrix)
scaled_matrix = scaler.transform(matrix)
In [207]:
pca = PCA()
pca.fit(scaled_matrix)
pca_samples = pca.transform(scaled_matrix)
In [213]:
fig, ax = plt.subplots(figsize=(50,20))
sns.set(font_scale=1)
plt.step(range(matrix.shape[1]), pca.explained_variance_ratio_.cumsum(), where='mid',
label='cumulative explained variance')
sns.barplot(np.arange(1,matrix.shape[1]+1), pca.explained_variance_ratio_, alpha=0.5, color = 'g',
label='individual explained variance')
plt.xlim(0, 50)
ax.set_xticklabels([s if int(s.get_text())%2 == 0 else '' for s in ax.get_xticklabels()])
plt.ylabel('Explained variance', fontsize = 50)
plt.xlabel('Principal components', fontsize = 50)
plt.legend(loc='best', fontsize = 50);
In [214]:
#Principle component analysis
from sklearn.decomposition import PCA
pca = PCA(n_components=50)
pca.fit(X_train)
pca_X_train=pca.transform(X_train)
pca_X_test=pca.transform(X_test)
8 XG Boost ON PCA¶
In [217]:
X_train= pca_X_train
X_test= pca_X_test
xgb_model7 = XGBRegressor(subsample= 0.9, silent= 1,
nthread= 4, n_estimators= 500, min_child_weight= 4,
max_depth= 6, learning_rate= 0.05, colsample_bytree= 1)
xgb_model7.fit(X_train, y_train, early_stopping_rounds=5,
eval_set=[(X_test, y_test)], verbose=False)
y_train_pred7 = xgb_model7.predict(X_train)
y_pred7 = xgb_model7.predict(X_test)
print('Train r2 score: ', r2_score(y_train_pred7, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred7))
train_mse5 = mean_squared_error(y_train_pred7, y_train)
test_mse5 = mean_squared_error(y_pred7, y_test)
train_rmse5 = np.sqrt(train_mse5)
test_rmse5 = np.sqrt(test_mse5)
print('Train RMSE: %.4f' % train_rmse5)
print('Test RMSE: %.4f' % test_rmse5)
Test Data
In [237]:
print("test_demographic dataset has {} samples with {} features each.".format(*test_demo_dt.shape))
print ("test_payment dataset has {} samples with {} features each.".format(*test_payment_dt.shape))
- describing data
demographic data¶
In [238]:
test_demo_dt.head()
Out[238]:
In [239]:
test_demo_dt.columns
Out[239]:
In [240]:
test_demo_dt.dtypes
Out[240]:
payment data¶
In [241]:
test_payment_dt.head(30)
Out[241]:
In [242]:
test_payment_dt.dtypes
Out[242]:
Convert DOB into Age¶
In [243]:
test_demo_dt['DOB']=test_demo_dt['DOB'].astype('datetime64')
test_demo_dt['age'] = (pd.to_datetime('now') - test_demo_dt['DOB']).astype('<m8[Y]')
test_demo_dt['age'] = test_demo_dt['age'].astype('int')
remove DOB¶
In [244]:
test_demo_dt=test_demo_dt.drop(axis=1,columns='DOB')
combine two dataset on primary key that is CustomerID for doing further analysis¶
In [245]:
test_merge_dt=test_payment_dt.merge(test_demo_dt, left_on='CustomerID', right_on='CustomerID')
In [246]:
test_merge_dt.columns
Out[246]:
In [247]:
test_merge_dt.head()
Out[247]:
In [248]:
test_merge_dt.shape
Out[248]:
In [249]:
sample_file.shape
Out[249]:
In [250]:
test_merge_dt.head(5)
Out[250]:
In [251]:
test_merge_dt.info()
In [252]:
test_merge_dt.describe(include='all').T.sort_values("count")
Out[252]:
- Observation - ProfessionalLicensure,eCommerceAccount,NoOfProperties are categorical type so need to change its data types
- observation -CreditRiskScore is target columns and we need to remove before to build the clusters
In [253]:
print(test_merge_dt.ProfessionalLicensure.unique())
print(test_merge_dt.eCommerceAccount.unique())
print(test_merge_dt.NoOfProperties.unique())
In [254]:
test_merge_dt['ProfessionalLicensure']=test_merge_dt['ProfessionalLicensure'].astype('object')
test_merge_dt['eCommerceAccount']=test_merge_dt['eCommerceAccount'].astype('object')
test_merge_dt['NoOfProperties']=test_merge_dt['NoOfProperties'].astype('object')
In [255]:
test_merge_dt.Urban_Development.unique()
Out[255]:
In [256]:
test_merge_dt.describe(include=['O'])
Out[256]:
In [257]:
test_merge_dt.describe(include=['float32','float64','int64','int32'])
Out[257]:
check missing data¶
In [258]:
#function to find missing values
def miss_data(x):
total = x.isnull().sum().sort_values(ascending=False)
percent = (x.isnull().sum()/x.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data.index.name = 'column_names'
missing_data.reset_index(inplace=True)
sns.set(style="whitegrid")
sns.set(rc={'figure.figsize':(20,8.27)})
sns.barplot(missing_data.column_names,missing_data.Percent, alpha=0.9)
print(plt.title('missing data plot'))
print(plt.ylabel('percentage of missing data', fontsize=12))
print(plt.xlabel('column names', fontsize=12))
print(plt.show())
print(missing_data)
In [259]:
miss_data(x=test_merge_dt)
- there is not any NA's present
Feature engineering¶
In [260]:
test_merge_dt['Appraisal_value']=test_merge_dt['Starting_outstanding']/(test_merge_dt['Starting_Loan_to_Appraisedvalu_Percent']/100)
In [261]:
test_merge_dt['current_Appraisal_value']=test_merge_dt['Current_Outstanding']/(test_merge_dt['Current_Loan_to_Appraisedvalu_Percent']/100)
In [262]:
test_merge_dt['remaining_outstanding']= test_merge_dt['Starting_outstanding'] - test_merge_dt['Current_Outstanding']
In [263]:
test_merge_dt.columns
Out[263]:
MAke cluster on Test Data¶
In [ ]:
test_matrix=test_merge_dt.as_matrix()
kmeans2.fit_predict(test_matrix)
clusters=kmeans2.labels_
test_cluster_class=clusters.tolist()
Data Needs Grouping
In [264]:
test_merge_dt_1=test_merge_dt[['CustomerID','Current_Instalment_Sequence', 'Current_Outstanding','Current_Loan_to_Appraisedvalu_Percent','CurrentInterestrate',
'RealEstate_Current_Inflation','GDP', 'UnemploymentRate','current_Appraisal_value','remaining_outstanding']]
In [265]:
test_merge_dt_1.shape
Out[265]:
Data Don't Need Grouping
In [269]:
test_merge_dt_2=test_merge_dt[['CustomerID', 'Starting_Instalment','Maturity_Period','Asset_type',
'Urban_Development', 'Villa_House', 'Investment_SelfOccupied',
'Starting_outstanding', 'Starting_Loan_to_Appraisedvalu_Percent',
'StartingInterestrate', 'RealEstate_Starting_Inflation',
'age', 'Salary', 'ProfessionalLicensure',
'UtilitySpending', 'eCommerceAccount', 'SocialMediaAccount','Appraisal_value',
'NoOfProperties']]
In [270]:
test_merge_dt_2=test_merge_dt_2.drop_duplicates()# drop duplicates from dataset
In [271]:
test_merge_dt_2.shape
Out[271]:
Function To Group The Data
In [272]:
def feat_eng(data):
df = pd.DataFrame()
for col in data.columns:
if col in ['CustomerID']:
continue
df[col + '_mean'] = data.groupby(['CustomerID'])[col].mean()
df[col + '_median'] = data.groupby(['CustomerID'])[col].median()
df[col + '_max'] = data.groupby(['CustomerID'])[col].max()
df[col + '_min'] = data.groupby(['CustomerID'])[col].min()
df[col + '_std'] = data.groupby(['CustomerID'])[col].std()
#df[col + '_kurtosis'] = data.groupby(['CustomerID'])[col].kurtosis()
df[col + '_skew'] = data.groupby(['CustomerID'])[col].skew()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_kurtosis'] = data.groupby(['CustomerID'])[col].apply(lambda x: x.kurtosis())
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
#df[col + '_coeffvar'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.std(x) / np.mean(x))
#in statistics, the median absolute deviation (MAD) is a robust measure of the variablility of a univariate sample of quantitative data.
df[col + '_meanAD'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_mad'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.median(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.max(np.abs(x)))
df[col + '_abs_min'] = data.groupby(['CustomerID'])[col].apply(lambda x: np.min(np.abs(x)))
df[col + '_abs_avg'] = (df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df
In [273]:
test_final_dt_1=feat_eng(data=test_merge_dt_1)
In [274]:
test_final_dt_1.head(10)
Out[274]:
In [275]:
test_final_dt_1['CustomerID']=test_final_dt_1.index
In [276]:
test_final_dt_1.head(10)
Out[276]:
In [277]:
test_final_dt_1.shape
Out[277]:
In [278]:
test_final_dt_1.describe(include='all')
Out[278]:
Join Two Column test_Merge dt 2 And test_final dt 1
In [279]:
test_train_data_1=test_final_dt_1.merge(test_merge_dt_2, left_on="CustomerID", right_on='CustomerID')
In [280]:
test_train_data_1.fillna(test_train_data_1['remaining_outstanding_maxtoMin'].median(),inplace=True)
In [281]:
test_train_data_1.isna().sum()
Out[281]:
In [282]:
test_train_data_1[['eCommerceAccount','ProfessionalLicensure']]=test_train_data_1[['eCommerceAccount','ProfessionalLicensure']].astype('object')
In [283]:
#num1_cols = list(train_data_1.select_dtypes(include=['float64','float32','int32','int64']).columns)
#cat1_cols = list(train_data_1.select_dtypes(include=['object']).columns)
categorical_features =test_train_data_1.select_dtypes(include=['object']).columns
numerical_features =test_train_data_1.select_dtypes(include=['float64','float32','int32','int64']).columns
Feature Engg On Payment Status Column
In [284]:
new=test_merge_dt[['CustomerID','Current_Instalment_Sequence', 'Current_Outstanding','Current_Loan_to_Appraisedvalu_Percent','CurrentInterestrate',
'RealEstate_Current_Inflation','GDP', 'UnemploymentRate','current_Appraisal_value','remaining_outstanding','Payment_Status']]
In [285]:
new.Payment_Status.unique()
Out[285]:
In [286]:
payment={'Non-Payoff/Non-Default': 2,'Payoff':4,'Default':0}
new['Payment_Status']=new['Payment_Status'].map(payment)
create payment_total_score¶
In [287]:
new['payment_total_score'] =[4]*new.shape[0]
create payment dataframe¶
In [288]:
payment_data=pd.DataFrame({'CustomerID':new.CustomerID,'Payment_Status':new.Payment_Status,'payment_total_score':new.payment_total_score})
define function to find z score of payment status¶
In [289]:
def payment_eng1(data):
df = pd.DataFrame()
for col in data.columns:
if col in ['CustomerID']:
continue
df[col + '_count'] = data.groupby(['CustomerID'])[col].count()
df[col + '_sum'] = data.groupby(['CustomerID'])[col].sum()
return df
In [290]:
payment_data=payment_eng1(data=payment_data)
In [291]:
payment_data.head(5)
Out[291]:
In [292]:
from scipy.stats import zscore
payment_percentile= payment_data.Payment_Status_sum/payment_data.payment_total_score_sum
payment_data['payment_z_score']=zscore(payment_percentile)
payment_data['payment_lenght']=payment_data.Payment_Status_count
In [293]:
print('size of train data',train_data_1.shape)
print('size of payment col',len(payment_percentile))
In [294]:
payment_data2=payment_data[['payment_z_score','payment_lenght']]
In [295]:
test_train_data_1=test_train_data_1.merge(payment_data2, left_on="CustomerID", right_on='CustomerID')
In [296]:
test_train_data_1.head(5)
Out[296]:
Skewed Transformation¶
In [299]:
#log transform skewed numeric features:
numeric_feats = test_train_data_1.dtypes[test_train_data_1.dtypes != "object"].index
skewed_feats = test_train_data_1[numeric_feats].apply(lambda x: skew(x.dropna())) #compute skewness
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
test_train_data_1[skewed_feats] = np.log1p(test_train_data_1[skewed_feats])
In [300]:
test_train_data_1[numeric_feats] = test_train_data_1[numeric_feats].apply(lambda x:pd.to_numeric(x)) #
In [301]:
test_train_data_1.drop(axis=1,columns='CustomerID',inplace=True)
In [302]:
test_df=test_train_data_1
In [304]:
test_df.head(5)
Out[304]:
In [306]:
test_df = pd.get_dummies(test_df,drop_first=True)
test_df = test_df.fillna(test_df.mean())
In [307]:
test_df.shape
Out[307]:
prediction¶
In [308]:
submission1= pd.read_csv('samplesubmission-1557545918238.csv')
test_dtt=test_df[most_relevant_features]
submission1.head(5)
predict_lmTest_ss = xgb_model6.predict(test_dtt)
predict_test=np.exp(predict_lmTest_ss)
submission1['CreditRiskScore'] = predict_test
submission1.head()
submission1.to_csv('xgb22.csv', index=False)
Conclusion¶
| Model | Validation | Test |
|---|---|---|
| XG Boost5 | 0.13 | 113.55 |
| MLR | 0.15 | |
| Elastic net | 0.16 | |
| XG Boost4 | 0.1355 | 117.35 |
| CAT Boost | 0.13 | 168.55 |
| KNN | 0.16 | 119.79 |
| Stacking | 0.1291 | 116.6 |
| XG boost(pca) | 0.1311 |
XG Boost perform better than others on this dataset